- name: d8.istio.multicluster
  rules:
    - alert: D8IstioMulticlusterRemoteAPIHostDoesntWork
      expr: max by (multicluster_name, api_host) (d8_istio_multicluster_api_host_check_error_count == 1)
      for: 5m
      labels:
        severity_level: "6"
        tier: cluster
      annotations:
        plk_markup_format: "markdown"
        plk_protocol_version: "1"
        plk_create_group_if_not_exists__d8_istio_multicluster_remote_api_host_failed: D8IstioMulticlusterRemoteAPIHostFailed,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes
        plk_grouped_by__d8_istio_multicluster_remote_api_host_failed: D8IstioMulticlusterRemoteAPIHostFailed,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes
        description: |
          Remote api host `{{$labels.api_host}}` for IstioMulticluster `{{$labels.multicluster_name}}` has failed healthcheck by d8 monitoring hook.

          Reproducing (run from deckhouse pod):
          ```
          TOKEN="$(deckhouse-controller module values istio -o json | jq -r --arg ah {{$labels.api_host}} '.istio.internal.multiclusters[] | select(.apiHost == $ah) | .apiJWT')"
          curl -H "Authorization: Bearer $TOKEN" https://{{$labels.api_host}}/version
          ```
        summary: Multicluster remote api host failed
    - alert: D8IstioMulticlusterMetadataEndpointDoesntWork
      expr: max by (multicluster_name, endpoint) (d8_istio_multicluster_metadata_endpoints_fetch_error_count == 1)
      for: 5m
      labels:
        severity_level: "6"
        tier: cluster
      annotations:
        plk_markup_format: "markdown"
        plk_protocol_version: "1"
        plk_create_group_if_not_exists__d8_istio_multicluster_metadata_endpoint_failed: D8IstioMulticlusterMetadataEndpointFailed,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes
        plk_grouped_by__d8_istio_multicluster_metadata_endpoint_failed: D8IstioMulticlusterMetadataEndpointFailed,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes
        description: |
          Metadata endpoint `{{$labels.endpoint}}` for IstioMulticluster `{{$labels.multicluster_name}}` has failed to fetch by d8 hook.
          Reproducing request to public endpoint:
          ```
          curl {{$labels.endpoint}}
          ```
          Reproducing request to private endpoints (run from deckhouse pod):
          ```
          KEY="$(deckhouse-controller module values istio -o json | jq -r .internal.remoteAuthnKeypair.priv)"
          LOCAL_CLUSTER_UUID="$(deckhouse-controller module values -g istio -o json | jq -r .global.discovery.clusterUUID)"
          REMOTE_CLUSTER_UUID="$(kubectl get istiomulticluster {{$labels.multicluster_name}} -o json | jq -r .status.metadataCache.public.clusterUUID)"
          TOKEN="$(deckhouse-controller helper gen-jwt --private-key-path <(echo "$KEY") --claim iss=d8-istio --claim sub=$LOCAL_CLUSTER_UUID --claim aud=$REMOTE_CLUSTER_UUID --claim scope=private-multicluster --ttl 1h)"
          curl -H "Authorization: Bearer $TOKEN" {{$labels.endpoint}}
          ```
        summary: Multicluster metadata endpoint failed
